# 合并所有文件，并移除重复的内容
import os
cwd = os.getcwd()
cdr = os.listdir(cwd)
lta = []
for fm in cdr:
    if '-11.txt' in fm:
        lta.append(fm)
print(lta)
lista = []
for fm1 in lta:
    with open(fm1, 'r', encoding='utf-8', errors='ignore') as fo:
        lines = fo.readlines()
        endline = len(lines)
        # print(fm1)
        for i in range(0, endline, 5):
            title = lines[i]
            urlt = lines[i+1]
            cnt = lines[i+2]
            if '.cn' in urlt:
                str1 = title+urlt+cnt+'----\n'
                if str1 in lista:
                    continue
                else:
                    lista.append(str1)
fm2 = '10-12.txt'
content = '\n'.join(lista)
with open(fm2, 'w', encoding='utf-8') as fd:
    fd.write(content)
